{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7fdbe3b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| default_exp prepare_t2s_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6cf56fcb",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ecbdddfd",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "import sys\n",
    "import os\n",
    "import itertools\n",
    "from pathlib import Path\n",
    "\n",
    "import numpy as np\n",
    "import torch\n",
    "import torchaudio\n",
    "import torch.nn.functional as F\n",
    "from torch.profiler import profile, record_function, ProfilerActivity\n",
    "\n",
    "from fastprogress import progress_bar\n",
    "from fastcore.script import *\n",
    "\n",
    "import whisper, whisperx\n",
    "from whisperspeech import vad, wh_transcribe, vq_stoks, extract_acoustic\n",
    "import webdataset as wds"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e1d80d3b",
   "metadata": {},
   "source": [
    "# T2S dataset preparation\n",
    "\n",
    "We take a webdataset shard and extract semantic tokens and transcriptions from it.\n",
    "\n",
    "We use VAD chunks merged with randomized maximum length to also generate some short samples."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "70522ef2",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| exporti\n",
    "def flac_to_t2s_name(input):\n",
    "    return input.rsplit(\"/\", 1)[1].replace('flac', 't2s') + \".gz\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3e5cc115",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'librilight-large-6454-t2s-000000.tar.gz'"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "flac_to_t2s_name('/data/whisperspeech-wds/librilight-large-6454-flac-000000.tar')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ac8bf372",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| exporti\n",
    "class Transcriber:\n",
    "    \"\"\"\n",
    "    A helper class to transcribe a batch of 30 second audio chunks.\n",
    "    \"\"\"\n",
    "    def __init__(self, model_size, lang=False):\n",
    "        self.model = whisperx.asr.load_model(model_size, \"cuda\", compute_type=\"float16\", language=lang)\n",
    "        # without calling vad_model at least once the rest segfaults for some reason...\n",
    "        self.model.vad_model({\"waveform\": torch.zeros(1, 16000), \"sample_rate\": 16000})\n",
    "        \n",
    "    def transcribe(self, batch):\n",
    "        batch = whisper.log_mel_spectrogram(batch)\n",
    "        embs = self.model.model.encode(batch.cpu().numpy())\n",
    "        return self.model.tokenizer.tokenizer.decode_batch([x.sequences_ids[0] for x in \n",
    "            self.model.model.model.generate(\n",
    "                embs,\n",
    "                [self.model.model.get_prompt(self.model.tokenizer, [], without_timestamps=True)]*len(batch),\n",
    "            )])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5f271d55",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| exporti\n",
    "@call_parse\n",
    "def prepare_t2s(\n",
    "    input:str,  # FLAC webdataset file path (or - to read the names from stdin)\n",
    "    proc_dataset_path:Path, # processed VAD files path\n",
    "    output:str=None, # output file name\n",
    "    vq_model:str=\"collabora/spear-tts-pytorch:whisper-vq-stoks.model\", # the model path (use repo_id:filename to download it from hugginface)\n",
    "    n_samples:int=None, # process a limited amount of samples\n",
    "    batch_size:int=1, # process several segments at once\n",
    "    transcription_model:str=\"small.en\",\n",
    "):\n",
    "    if \":\" in vq_model:\n",
    "        repo, fname = vq_model.split(\":\", 1)\n",
    "        vq_model = vq_stoks.RQBottleneckTransformer.load_model(repo, fname).cuda()\n",
    "    else:\n",
    "        vq_model = vq_stoks.RQBottleneckTransformer.load_model(local_filename=vq_model).cuda()\n",
    "    transcriber = Transcriber(transcription_model)\n",
    "        \n",
    "    if input == \"-\":\n",
    "        input = [f.strip() for f in sys.stdin.readlines()]\n",
    "        assert output, \"please provide the output shard name\"\n",
    "    else:\n",
    "        if output is None: output = flac_to_t2s_name(input)\n",
    "        input = [input]\n",
    "        \n",
    "    total = n_samples//batch_size if n_samples else 'noinfer'\n",
    "    if n_samples: print(f\"Benchmarking run of {n_samples} samples ({total} batches)\")\n",
    "\n",
    "    ds = wds.WebDataset(input, shardshuffle=True, rename_files=vad.fix_dots_in_names).compose(\n",
    "        wds.decode(wds.torch_audio),\n",
    "        vq_stoks.merge_in(vq_stoks.derived_dataset(proc_dataset_path, 'vad')),\n",
    "        wds.map_dict(**{\"vad.npy\": lambda s: wh_transcribe.chunk_merger(s, wh_transcribe.random_cutter)}),\n",
    "        lambda x: wh_transcribe.split_to_chunks(x),\n",
    "        # drop the first and last segment because they tend to be inaccurate\n",
    "        # (the transcriptions don't have the \"LibriVox\" header and \"end of chapter\" suffix)\n",
    "        wds.select(lambda x: x['i'] != 0 and x['i'] != x['imax']),\n",
    "        wds.to_tuple('__key__', 'rpad', 'samples'),\n",
    "        wds.batched(64),\n",
    "    )\n",
    "\n",
    "    dl = wds.WebLoader(ds, num_workers=4, batch_size=None).unbatched().shuffle(2000).batched(batch_size)\n",
    "\n",
    "    speakers = set()\n",
    "    tmp = output+\".tmp\"\n",
    "    with wds.TarWriter(tmp) as sink:\n",
    "        for keys, rpads, samples in progress_bar(dl, total=total):\n",
    "            with record_function('to_cuda'):\n",
    "                csamples = samples.cuda()\n",
    "            with record_function('transcribe'):\n",
    "                txts = transcriber.transcribe(csamples)\n",
    "            with record_function('vq_stoks'):\n",
    "                stoks = vq_model.encode_audio(csamples)\n",
    "            with record_function('from_cuda'):\n",
    "                stoks = stoks.cpu().numpy().astype(np.int16)\n",
    "            for key, rpad, txt, _stoks in zip(keys, rpads, txts, stoks):\n",
    "                speakers.add(key.split('/')[1])\n",
    "                sink.write({\n",
    "                    \"__key__\": key,\n",
    "                    \"txt\": txt,\n",
    "                    \"stoks.npy\": _stoks[:int(-rpad/16000 * 25)],\n",
    "                })\n",
    "    with open(output+\".speakers.txt\", \"w\") as f: f.write(\"\\n\".join(speakers))\n",
    "    if not n_samples:\n",
    "        os.rename(tmp, output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "034587a5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<style>\n",
       "    /* Turns off some styling */\n",
       "    progress {\n",
       "        /* gets rid of default border in Firefox and Opera. */\n",
       "        border: none;\n",
       "        /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "        background-size: auto;\n",
       "    }\n",
       "    progress:not([value]), progress:not([value])::-webkit-progress-bar {\n",
       "        background: repeating-linear-gradient(45deg, #7e7e7e, #7e7e7e 10px, #5c5c5c 10px, #5c5c5c 20px);\n",
       "    }\n",
       "    .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "        background: #F44336;\n",
       "    }\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      <progress value='232' class='' max='232' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [232/232 00:47&lt;00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "85.79258390624999"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds = wds.WebDataset(['/data/whisperspeech-wds/librilight-large-6454-flac-000000.tar'], rename_files=vad.fix_dots_in_names).compose(wds.decode(wds.torch_audio))\n",
    "sum([x['flac'][0].shape[-1]/16000/3600 for x in progress_bar(ds, total='noinfer')])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1ab9d59f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<style>\n",
       "    /* Turns off some styling */\n",
       "    progress {\n",
       "        /* gets rid of default border in Firefox and Opera. */\n",
       "        border: none;\n",
       "        /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "        background-size: auto;\n",
       "    }\n",
       "    progress:not([value]), progress:not([value])::-webkit-progress-bar {\n",
       "        background: repeating-linear-gradient(45deg, #7e7e7e, #7e7e7e 10px, #5c5c5c 10px, #5c5c5c 20px);\n",
       "    }\n",
       "    .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "        background: #F44336;\n",
       "    }\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      <progress value='16479' class='' max='16479' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [16479/16479 00:59&lt;00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "tensor([-15.7778, -14.8559, -13.3381,  ...,  -0.0186,   0.1787,   0.2196])"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds = wds.WebDataset(['/data/whisperspeech-wds/librilight-large-6454-flac-000000.tar'], rename_files=vad.fix_dots_in_names).compose(\n",
    "    wds.decode(wds.torch_audio),\n",
    "    vq_stoks.merge_in(vq_stoks.derived_dataset('/data/whisperspeech-processed-wds/', 'vad')),\n",
    "    wds.map_dict(**{\"vad.npy\": lambda s: wh_transcribe.chunk_merger(s, wh_transcribe.random_cutter)}),\n",
    "    lambda x: wh_transcribe.split_to_chunks(x),\n",
    "    # drop the first and last segment because they tend to be inaccurate\n",
    "    # (the transcriptions don't have the \"LibriVox\" header and \"end of chapter\" suffix)\n",
    "    wds.select(lambda x: x['i'] != 0 and x['i'] != x['imax']),\n",
    ")\n",
    "sum([x['samples'].shape[-1] for x in progress_bar(ds, total='noinfer')])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ead390d7",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Lightning automatically upgraded your loaded checkpoint from v1.5.4 to v2.0.9. To apply the upgrade to your files permanently, run `python -m pytorch_lightning.utilities.upgrade_checkpoint --file ../../../.cache/torch/whisperx-vad-segmentation.bin`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model was trained with pyannote.audio 0.0.1, yours is 2.1.1. Bad things might happen unless you revert pyannote.audio to 0.x.\n",
      "Model was trained with torch 1.10.0+cu102, yours is 2.0.1+cu118. Bad things might happen unless you revert torch to 1.x.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "<style>\n",
       "    /* Turns off some styling */\n",
       "    progress {\n",
       "        /* gets rid of default border in Firefox and Opera. */\n",
       "        border: none;\n",
       "        /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "        background-size: auto;\n",
       "    }\n",
       "    progress:not([value]), progress:not([value])::-webkit-progress-bar {\n",
       "        background: repeating-linear-gradient(45deg, #7e7e7e, #7e7e7e 10px, #5c5c5c 10px, #5c5c5c 20px);\n",
       "    }\n",
       "    .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "        background: #F44336;\n",
       "    }\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      <progress value='15' class='' max='15' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [15/15 00:18&lt;00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "prepare_t2s('/data/whisperspeech-wds/librilight-large-6454-flac-000000.tar', '/data/whisperspeech-processed-wds/', vq_model='vqmodel-4e-hyptuned-32gpu.model', n_samples=500, batch_size=32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8caaae5d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Lightning automatically upgraded your loaded checkpoint from v1.5.4 to v2.0.9. To apply the upgrade to your files permanently, run `python -m pytorch_lightning.utilities.upgrade_checkpoint --file ../../../.cache/torch/whisperx-vad-segmentation.bin`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model was trained with pyannote.audio 0.0.1, yours is 2.1.1. Bad things might happen unless you revert pyannote.audio to 0.x.\n",
      "Model was trained with torch 1.10.0+cu102, yours is 2.0.1+cu118. Bad things might happen unless you revert torch to 1.x.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "<style>\n",
       "    /* Turns off some styling */\n",
       "    progress {\n",
       "        /* gets rid of default border in Firefox and Opera. */\n",
       "        border: none;\n",
       "        /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "        background-size: auto;\n",
       "    }\n",
       "    progress:not([value]), progress:not([value])::-webkit-progress-bar {\n",
       "        background: repeating-linear-gradient(45deg, #7e7e7e, #7e7e7e 10px, #5c5c5c 10px, #5c5c5c 20px);\n",
       "    }\n",
       "    .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "        background: #F44336;\n",
       "    }\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      <progress value='15' class='' max='15' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [15/15 00:22&lt;00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    }
   ],
   "source": [
    "prepare_t2s('/data/whisperspeech-wds/librilight-large-6454-flac-000000.tar', '/data/whisperspeech-processed-wds/', vq_model='vqmodel-4e-hyptuned-32gpu.model', n_samples=500, batch_size=32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ff7e835c",
   "metadata": {},
   "outputs": [],
   "source": [
    "## Batch size tests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0a3a8e60",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<style>\n",
       "    /* Turns off some styling */\n",
       "    progress {\n",
       "        /* gets rid of default border in Firefox and Opera. */\n",
       "        border: none;\n",
       "        /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "        background-size: auto;\n",
       "    }\n",
       "    progress:not([value]), progress:not([value])::-webkit-progress-bar {\n",
       "        background: repeating-linear-gradient(45deg, #7e7e7e, #7e7e7e 10px, #5c5c5c 10px, #5c5c5c 20px);\n",
       "    }\n",
       "    .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "        background: #F44336;\n",
       "    }\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      <progress value='1000' class='' max='1000' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [1000/1000 00:55&lt;00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "prepare_s2a('/data/whisperspeech-wds/librilight-large-6454-flac-000000.tar', '/data/whisperspeech-processed-wds/', vq_model='vqmodel-4e-hyptuned-32gpu.model', n_samples=1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "94e5010c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<style>\n",
       "    /* Turns off some styling */\n",
       "    progress {\n",
       "        /* gets rid of default border in Firefox and Opera. */\n",
       "        border: none;\n",
       "        /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "        background-size: auto;\n",
       "    }\n",
       "    progress:not([value]), progress:not([value])::-webkit-progress-bar {\n",
       "        background: repeating-linear-gradient(45deg, #7e7e7e, #7e7e7e 10px, #5c5c5c 10px, #5c5c5c 20px);\n",
       "    }\n",
       "    .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "        background: #F44336;\n",
       "    }\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      <progress value='500' class='' max='500' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [500/500 00:34&lt;00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "prepare_s2a('/data/whisperspeech-wds/librilight-large-6454-flac-000000.tar', '/data/whisperspeech-processed-wds/', vq_model='vqmodel-4e-hyptuned-32gpu.model', n_samples=1000, batch_size=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a192fc93",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<style>\n",
       "    /* Turns off some styling */\n",
       "    progress {\n",
       "        /* gets rid of default border in Firefox and Opera. */\n",
       "        border: none;\n",
       "        /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "        background-size: auto;\n",
       "    }\n",
       "    progress:not([value]), progress:not([value])::-webkit-progress-bar {\n",
       "        background: repeating-linear-gradient(45deg, #7e7e7e, #7e7e7e 10px, #5c5c5c 10px, #5c5c5c 20px);\n",
       "    }\n",
       "    .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "        background: #F44336;\n",
       "    }\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      <progress value='250' class='' max='250' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [250/250 00:24&lt;00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "prepare_s2a('/data/whisperspeech-wds/librilight-large-6454-flac-000000.tar', '/data/whisperspeech-processed-wds/', vq_model='vqmodel-4e-hyptuned-32gpu.model', n_samples=1000, batch_size=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "51180711",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<style>\n",
       "    /* Turns off some styling */\n",
       "    progress {\n",
       "        /* gets rid of default border in Firefox and Opera. */\n",
       "        border: none;\n",
       "        /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "        background-size: auto;\n",
       "    }\n",
       "    progress:not([value]), progress:not([value])::-webkit-progress-bar {\n",
       "        background: repeating-linear-gradient(45deg, #7e7e7e, #7e7e7e 10px, #5c5c5c 10px, #5c5c5c 20px);\n",
       "    }\n",
       "    .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "        background: #F44336;\n",
       "    }\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      <progress value='125' class='' max='125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [125/125 00:20&lt;00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "prepare_s2a('/data/whisperspeech-wds/librilight-large-6454-flac-000000.tar', '/data/whisperspeech-processed-wds/', vq_model='vqmodel-4e-hyptuned-32gpu.model', n_samples=1000, batch_size=8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bbde0eed",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<style>\n",
       "    /* Turns off some styling */\n",
       "    progress {\n",
       "        /* gets rid of default border in Firefox and Opera. */\n",
       "        border: none;\n",
       "        /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "        background-size: auto;\n",
       "    }\n",
       "    progress:not([value]), progress:not([value])::-webkit-progress-bar {\n",
       "        background: repeating-linear-gradient(45deg, #7e7e7e, #7e7e7e 10px, #5c5c5c 10px, #5c5c5c 20px);\n",
       "    }\n",
       "    .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "        background: #F44336;\n",
       "    }\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      <progress value='1000' class='' max='1000' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [1000/1000 00:27&lt;00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# stoks only\n",
    "prepare_s2a('/data/whisperspeech-wds/librilight-large-6454-flac-000000.tar', '/data/whisperspeech-processed-wds/', vq_model='vqmodel-4e-hyptuned-32gpu.model', n_samples=1000, batch_size=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "11dc80d5",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "STAGE:2023-10-06 14:25:45 71030:71030 ActivityProfilerController.cpp:311] Completed Stage: Warm Up\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "<style>\n",
       "    /* Turns off some styling */\n",
       "    progress {\n",
       "        /* gets rid of default border in Firefox and Opera. */\n",
       "        border: none;\n",
       "        /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "        background-size: auto;\n",
       "    }\n",
       "    progress:not([value]), progress:not([value])::-webkit-progress-bar {\n",
       "        background: repeating-linear-gradient(45deg, #7e7e7e, #7e7e7e 10px, #5c5c5c 10px, #5c5c5c 20px);\n",
       "    }\n",
       "    .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "        background: #F44336;\n",
       "    }\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      <progress value='10' class='' max='10' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [10/10 00:01&lt;00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "STAGE:2023-10-06 14:25:47 71030:71030 ActivityProfilerController.cpp:317] Completed Stage: Collection\n",
      "STAGE:2023-10-06 14:25:47 71030:71030 ActivityProfilerController.cpp:321] Completed Stage: Post Processing\n"
     ]
    }
   ],
   "source": [
    "# atoks only\n",
    "with profile(activities=[ProfilerActivity.CPU, ProfilerActivity.CUDA]) as prof:\n",
    "    prepare_s2a('/data/whisperspeech-wds/librilight-large-6454-flac-000000.tar', '/data/whisperspeech-processed-wds/', vq_model='vqmodel-4e-hyptuned-32gpu.model', n_samples=10, batch_size=1)\n",
    "prof.export_chrome_trace(\"trace-bs1.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0b0250d3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-rw-r--r-- 1 root root 1.3M Oct  6 09:17 librilight-large-6454-s2a-000000.tar.gz.tmp\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_01_kipling_64kb_000.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_01_kipling_64kb_000.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_01_kipling_64kb_001.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_01_kipling_64kb_001.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_01_kipling_64kb_002.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_01_kipling_64kb_002.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_01_kipling_64kb_003.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_01_kipling_64kb_003.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_01_kipling_64kb_004.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_01_kipling_64kb_004.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_01_kipling_64kb_005.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_01_kipling_64kb_005.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_01_kipling_64kb_006.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_01_kipling_64kb_006.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_01_kipling_64kb_007.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_01_kipling_64kb_007.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_01_kipling_64kb_008.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_01_kipling_64kb_008.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_01_kipling_64kb_009.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_01_kipling_64kb_009.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_01_kipling_64kb_010.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_01_kipling_64kb_010.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_01_kipling_64kb_011.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_01_kipling_64kb_011.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_01_kipling_64kb_012.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_01_kipling_64kb_012.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_01_kipling_64kb_013.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_01_kipling_64kb_013.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_01_kipling_64kb_014.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_01_kipling_64kb_014.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_01_kipling_64kb_015.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_01_kipling_64kb_015.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_01_kipling_64kb_016.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_01_kipling_64kb_016.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_01_kipling_64kb_017.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_01_kipling_64kb_017.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_01_kipling_64kb_018.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_01_kipling_64kb_018.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_01_kipling_64kb_019.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_01_kipling_64kb_019.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_01_kipling_64kb_020.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_01_kipling_64kb_020.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_01_kipling_64kb_021.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_01_kipling_64kb_021.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_01_kipling_64kb_022.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_01_kipling_64kb_022.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_01_kipling_64kb_023.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_01_kipling_64kb_023.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_01_kipling_64kb_024.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_01_kipling_64kb_024.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_01_kipling_64kb_025.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_01_kipling_64kb_025.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_01_kipling_64kb_026.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_01_kipling_64kb_026.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_02_kipling_64kb_000.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_02_kipling_64kb_000.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_02_kipling_64kb_001.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_02_kipling_64kb_001.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_02_kipling_64kb_002.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_02_kipling_64kb_002.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_02_kipling_64kb_003.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_02_kipling_64kb_003.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_02_kipling_64kb_004.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_02_kipling_64kb_004.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_02_kipling_64kb_005.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_02_kipling_64kb_005.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_02_kipling_64kb_006.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_02_kipling_64kb_006.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_02_kipling_64kb_007.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_02_kipling_64kb_007.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_02_kipling_64kb_008.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_02_kipling_64kb_008.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_02_kipling_64kb_009.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_02_kipling_64kb_009.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_02_kipling_64kb_010.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_02_kipling_64kb_010.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_02_kipling_64kb_011.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_02_kipling_64kb_011.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_02_kipling_64kb_012.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_02_kipling_64kb_012.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_02_kipling_64kb_013.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_02_kipling_64kb_013.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_02_kipling_64kb_014.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_02_kipling_64kb_014.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_02_kipling_64kb_015.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_02_kipling_64kb_015.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_02_kipling_64kb_016.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_02_kipling_64kb_016.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_02_kipling_64kb_017.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_02_kipling_64kb_017.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_02_kipling_64kb_018.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_02_kipling_64kb_018.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_02_kipling_64kb_019.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_02_kipling_64kb_019.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_02_kipling_64kb_020.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_02_kipling_64kb_020.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_03_kipling_64kb_000.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_03_kipling_64kb_000.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_03_kipling_64kb_001.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_03_kipling_64kb_001.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_03_kipling_64kb_002.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_03_kipling_64kb_002.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_03_kipling_64kb_003.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_03_kipling_64kb_003.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_03_kipling_64kb_004.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_03_kipling_64kb_004.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_03_kipling_64kb_005.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_03_kipling_64kb_005.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_03_kipling_64kb_006.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_03_kipling_64kb_006.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_03_kipling_64kb_007.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_03_kipling_64kb_007.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_03_kipling_64kb_008.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_03_kipling_64kb_008.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_03_kipling_64kb_009.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_03_kipling_64kb_009.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_03_kipling_64kb_010.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_03_kipling_64kb_010.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_03_kipling_64kb_011.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_03_kipling_64kb_011.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_03_kipling_64kb_012.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_03_kipling_64kb_012.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_03_kipling_64kb_013.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_03_kipling_64kb_013.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_03_kipling_64kb_014.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_03_kipling_64kb_014.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_04_kipling_64kb_000.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_04_kipling_64kb_000.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_04_kipling_64kb_001.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_04_kipling_64kb_001.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_04_kipling_64kb_002.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_04_kipling_64kb_002.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_04_kipling_64kb_003.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_04_kipling_64kb_003.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_04_kipling_64kb_004.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_04_kipling_64kb_004.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_04_kipling_64kb_005.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_04_kipling_64kb_005.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_04_kipling_64kb_006.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_04_kipling_64kb_006.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_04_kipling_64kb_007.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_04_kipling_64kb_007.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_04_kipling_64kb_008.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_04_kipling_64kb_008.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_04_kipling_64kb_009.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_04_kipling_64kb_009.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_04_kipling_64kb_010.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_04_kipling_64kb_010.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_04_kipling_64kb_011.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_04_kipling_64kb_011.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_04_kipling_64kb_012.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_04_kipling_64kb_012.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_04_kipling_64kb_013.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_04_kipling_64kb_013.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_04_kipling_64kb_014.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_04_kipling_64kb_014.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_04_kipling_64kb_015.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_04_kipling_64kb_015.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_04_kipling_64kb_016.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_04_kipling_64kb_016.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_05_kipling_64kb_000.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_05_kipling_64kb_000.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_05_kipling_64kb_001.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_05_kipling_64kb_001.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_05_kipling_64kb_002.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_05_kipling_64kb_002.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_05_kipling_64kb_003.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_05_kipling_64kb_003.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_05_kipling_64kb_004.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_05_kipling_64kb_004.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_05_kipling_64kb_005.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_05_kipling_64kb_005.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_05_kipling_64kb_006.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_05_kipling_64kb_006.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_05_kipling_64kb_007.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_05_kipling_64kb_007.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_05_kipling_64kb_008.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_05_kipling_64kb_008.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_05_kipling_64kb_009.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_05_kipling_64kb_009.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_05_kipling_64kb_010.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_05_kipling_64kb_010.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_05_kipling_64kb_011.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_05_kipling_64kb_011.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_05_kipling_64kb_012.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_05_kipling_64kb_012.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_05_kipling_64kb_013.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_05_kipling_64kb_013.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_05_kipling_64kb_014.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_05_kipling_64kb_014.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_05_kipling_64kb_015.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_05_kipling_64kb_015.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_05_kipling_64kb_016.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_05_kipling_64kb_016.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_05_kipling_64kb_017.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_05_kipling_64kb_017.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_05_kipling_64kb_018.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_05_kipling_64kb_018.stoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_06_kipling_64kb_000.atoks.npy\n",
      "large/6454/abaft_funnel_1307_librivox_64kb_mp3/funnel_06_kipling_64kb_000.stoks.npy\n"
     ]
    }
   ],
   "source": [
    "!ls -lh librilight-large-6454-s2a-000000.tar.gz.tmp\n",
    "!tar -tf librilight-large-6454-s2a-000000.tar.gz.tmp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "68b66e6f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| hide\n",
    "import nbdev; nbdev.nbdev_export()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "68ace212",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "python3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
